{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-02-26T02:34:00.021293Z",
     "start_time": "2025-02-26T02:33:58.690744Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "tensor([[0.2787],\n        [0.2679]], grad_fn=<AddmmBackward0>)"
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "net = nn.Sequential(nn.Linear(4, 8), nn.ReLU(), nn.Linear(8, 1))\n",
    "X = torch.rand(size=(2, 4))\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "\u001B[1;35mOrderedDict\u001B[0m\u001B[1m(\u001B[0m\u001B[1m[\u001B[0m\u001B[1m(\u001B[0m\u001B[32m'weight'\u001B[0m, \u001B[1;35mtensor\u001B[0m\u001B[1m(\u001B[0m\u001B[1m[\u001B[0m\u001B[1m[\u001B[0m\u001B[1;36m-0.3183\u001B[0m, \u001B[1;36m-0.2029\u001B[0m, \u001B[1;36m-0.3074\u001B[0m, \u001B[1;36m-0.2694\u001B[0m, \u001B[1;36m-0.0014\u001B[0m,  \u001B[1;36m0.1515\u001B[0m, \u001B[1;36m-0.3520\u001B[0m, \u001B[1;36m-0.0138\u001B[0m\u001B[1m]\u001B[0m\u001B[1m]\u001B[0m\u001B[1m)\u001B[0m\u001B[1m)\u001B[0m, \n\u001B[1m(\u001B[0m\u001B[32m'bias'\u001B[0m, \u001B[1;35mtensor\u001B[0m\u001B[1m(\u001B[0m\u001B[1m[\u001B[0m\u001B[1;36m-0.1768\u001B[0m\u001B[1m]\u001B[0m\u001B[1m)\u001B[0m\u001B[1m)\u001B[0m\u001B[1m]\u001B[0m\u001B[1m)\u001B[0m\n",
      "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">OrderedDict</span><span style=\"font-weight: bold\">([(</span><span style=\"color: #008000; text-decoration-color: #008000\">'weight'</span>, <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3183</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2029</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3074</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2694</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0014</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1515</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3520</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0138</span><span style=\"font-weight: bold\">]]))</span>, \n<span style=\"font-weight: bold\">(</span><span style=\"color: #008000; text-decoration-color: #008000\">'bias'</span>, <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1768</span><span style=\"font-weight: bold\">]))])</span>\n</pre>\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 检查第二个全连接层的参数\n",
    "# !pip install rich\n",
    "\n",
    "# from rich import print\n",
    "print(net[2].state_dict())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T02:32:15.574536Z",
     "start_time": "2025-02-26T02:32:15.508589Z"
    }
   },
   "id": "9b50bd53b1afcce4",
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('weight', tensor([[-0.0977,  0.1467, -0.0210,  0.3175,  0.0123, -0.1483,  0.2682,  0.0210]])), ('bias', tensor([0.1079]))])\n"
     ]
    }
   ],
   "source": [
    "print(net[2].state_dict())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T02:34:08.228877Z",
     "start_time": "2025-02-26T02:34:08.223714Z"
    }
   },
   "id": "e49d41f762b4658b",
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.nn.parameter.Parameter'>\n",
      "Parameter containing:\n",
      "tensor([0.1079], requires_grad=True)\n",
      "tensor([0.1079])\n"
     ]
    }
   ],
   "source": [
    "print(type(net[2].bias))\n",
    "print(net[2].bias)\n",
    "print(net[2].bias.data)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T03:49:16.506134Z",
     "start_time": "2025-02-26T03:49:16.500605Z"
    }
   },
   "id": "1cdc521082fa387e",
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "True"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[2].weight.grad == None"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T03:50:54.334526Z",
     "start_time": "2025-02-26T03:50:54.329576Z"
    }
   },
   "id": "8b7a55086f135742",
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('weight', torch.Size([8, 4])) ('bias', torch.Size([8]))\n",
      "('0.weight', torch.Size([8, 4])) ('0.bias', torch.Size([8])) ('2.weight', torch.Size([1, 8])) ('2.bias', torch.Size([1]))\n"
     ]
    }
   ],
   "source": [
    "print(*[(name, param.shape) for name, param in net[0].named_parameters()])\n",
    "print(*[(name, param.shape) for name, param in net.named_parameters()])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T03:53:18.706537Z",
     "start_time": "2025-02-26T03:53:18.702204Z"
    }
   },
   "id": "9923bcf200e56727",
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('weight', Parameter containing:\n",
      "tensor([[ 0.4669, -0.0379, -0.2797,  0.5624],\n",
      "        [-0.1977, -0.4269,  0.4705,  0.3729],\n",
      "        [ 0.2983,  0.5809, -0.1385, -0.3316],\n",
      "        [ 0.6002, -0.5691, -0.4946, -0.2450],\n",
      "        [-0.3612,  0.3631, -0.2457, -0.2810],\n",
      "        [-0.4958, -0.1253, -0.5021, -0.6657],\n",
      "        [-0.4521, -0.3549,  0.2242,  0.6336],\n",
      "        [-0.6031,  0.5207, -0.1953,  0.1424]], requires_grad=True)), ('bias', Parameter containing:\n",
      "tensor([0., 0., 0., 0., 0., 0., 0., 0.], requires_grad=True))]\n"
     ]
    }
   ],
   "source": [
    "print(list(net[0].named_parameters()))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T05:19:55.357961Z",
     "start_time": "2025-02-26T05:19:55.352947Z"
    }
   },
   "id": "6dfd8417efe768e",
   "execution_count": 16
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "OrderedDict([('0.weight',\n              tensor([[-7.7959e-02,  3.4669e-01, -4.2020e-01,  1.6643e-02],\n                      [-4.5816e-01,  4.7317e-01, -4.1316e-01,  3.0751e-01],\n                      [ 4.6531e-01,  1.2197e-02, -1.0118e-01,  2.2875e-01],\n                      [-2.1396e-01,  2.6607e-03, -4.1192e-01, -3.8162e-01],\n                      [-3.7456e-04, -1.8765e-01, -3.2839e-01,  4.8873e-01],\n                      [-4.9039e-01,  4.0014e-01,  3.6348e-01, -4.7209e-01],\n                      [ 4.1754e-01, -3.7955e-01,  3.8895e-01,  4.0714e-01],\n                      [ 3.8959e-01,  2.5921e-01,  2.0238e-01,  4.1929e-01]])),\n             ('0.bias',\n              tensor([-0.4650, -0.1139,  0.2986,  0.2160, -0.1251,  0.1633,  0.0814,  0.3718])),\n             ('2.weight',\n              tensor([[-0.0977,  0.1467, -0.0210,  0.3175,  0.0123, -0.1483,  0.2682,  0.0210]])),\n             ('2.bias', tensor([0.1079]))])"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.state_dict()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T03:54:50.682673Z",
     "start_time": "2025-02-26T03:54:50.675543Z"
    }
   },
   "id": "c8016c1ecb63b5fd",
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "tensor([0.1079])"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.state_dict()['2.bias'].data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T03:55:02.230082Z",
     "start_time": "2025-02-26T03:55:02.225094Z"
    }
   },
   "id": "ae6ba709b9734512",
   "execution_count": 7
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 5.2.1.3. 从嵌套块收集参数"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "545b2b22b4e93061"
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "tensor([[-0.2889],\n        [-0.2890]], grad_fn=<AddmmBackward0>)"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def block1():\n",
    "    return nn.Sequential(nn.Linear(4, 8), nn.ReLU(),\n",
    "                         nn.Linear(8, 4), nn.ReLU())\n",
    "\n",
    "def block2():\n",
    "    net = nn.Sequential()\n",
    "    for i in range(4):\n",
    "        # 在这里嵌套\n",
    "        net.add_module(f'block {i}', block1())\n",
    "    return net\n",
    "\n",
    "rgnet = nn.Sequential(block2(), nn.Linear(4, 1))\n",
    "rgnet(X)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T03:57:09.706092Z",
     "start_time": "2025-02-26T03:57:09.695189Z"
    }
   },
   "id": "2826aa178fd5682",
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Sequential(\n",
      "    (block 0): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 1): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 2): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block 3): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "  )\n",
      "  (1): Linear(in_features=4, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(rgnet)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T03:57:16.130519Z",
     "start_time": "2025-02-26T03:57:16.126961Z"
    }
   },
   "id": "15df2461e82233fe",
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "tensor([ 0.1178,  0.3877,  0.1397,  0.4897,  0.2314,  0.0399, -0.2432,  0.4855])"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgnet[0][1][0].bias.data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T03:57:22.132006Z",
     "start_time": "2025-02-26T03:57:22.127237Z"
    }
   },
   "id": "b6e4bde9406ccc33",
   "execution_count": 10
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 5.2.2. 参数初始化"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "11e32c135a919614"
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "(tensor([ 0.0092, -0.0128,  0.0141, -0.0257]), tensor(0.))"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def init_normal(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.normal_(m.weight, mean=0, std=0.01)\n",
    "        nn.init.zeros_(m.bias)\n",
    "net.apply(init_normal)\n",
    "net[0].weight.data[0], net[0].bias.data[0]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T03:59:03.527033Z",
     "start_time": "2025-02-26T03:59:03.519494Z"
    }
   },
   "id": "e56407823ac3745f",
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "(tensor([1., 1., 1., 1.]), tensor(0.))"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def init_constant(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.constant_(m.weight, 1)\n",
    "        nn.init.zeros_(m.bias)\n",
    "net.apply(init_constant)\n",
    "net[0].weight.data[0], net[0].bias.data[0]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T03:59:41.023858Z",
     "start_time": "2025-02-26T03:59:41.017339Z"
    }
   },
   "id": "3783802eab1f1790",
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.4669, -0.0379, -0.2797,  0.5624])\n",
      "tensor([[42., 42., 42., 42., 42., 42., 42., 42.]])\n"
     ]
    }
   ],
   "source": [
    "def init_xavier(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.xavier_uniform_(m.weight)\n",
    "def init_42(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.constant_(m.weight, 42)\n",
    "\n",
    "net[0].apply(init_xavier)\n",
    "net[2].apply(init_42)\n",
    "print(net[0].weight.data[0])\n",
    "print(net[2].weight.data)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T04:00:26.057749Z",
     "start_time": "2025-02-26T04:00:26.052326Z"
    }
   },
   "id": "482092a5049e0f1f",
   "execution_count": 13
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 5.2.2.2. 自定义初始化\n",
    "\\begin{split}\\begin{aligned}\n",
    "    w \\sim \\begin{cases}\n",
    "        U(5, 10) & \\text{ 可能性 } \\frac{1}{4} \\\\\n",
    "            0    & \\text{ 可能性 } \\frac{1}{2} \\\\\n",
    "        U(-10, -5) & \\text{ 可能性 } \\frac{1}{4}\n",
    "    \\end{cases}\n",
    "\\end{aligned}\\end{split}"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "90088b5fca651ae9"
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init weight torch.Size([8, 4])\n",
      "Init weight torch.Size([1, 8])\n"
     ]
    },
    {
     "data": {
      "text/plain": "tensor([[-5.4061, -8.3863,  6.5925,  8.1013],\n        [-0.0000,  0.0000,  9.1406,  0.0000]], grad_fn=<SliceBackward0>)"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def my_init(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        print(\"Init\", *[(name, param.shape)\n",
    "                        for name, param in m.named_parameters()][0])\n",
    "        nn.init.uniform_(m.weight, -10, 10)\n",
    "        m.weight.data *= m.weight.data.abs() >= 5\n",
    "\n",
    "net.apply(my_init)\n",
    "net[0].weight[:2]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T05:25:37.015476Z",
     "start_time": "2025-02-26T05:25:36.998311Z"
    }
   },
   "id": "59deceb21d66354",
   "execution_count": 17
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "tensor([42.0000, -7.3863,  7.5925,  9.1013])"
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 直接设置参数\n",
    "net[0].weight.data[:] += 1\n",
    "net[0].weight.data[0, 0] = 42\n",
    "net[0].weight.data[0]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-02-26T05:30:50.853845Z",
     "start_time": "2025-02-26T05:30:50.843855Z"
    }
   },
   "id": "d92ed259118cbfd1",
   "execution_count": 18
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "ccf9735cbed6d7a"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
